#!/usr/bin/env python3
from do_get_aminer_data import *
from config import *
import pymongo
import pandas as pd
from multiprocessing import Pool
import json
import codecs

#配置mongodb的信息
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_RESEARCHER_SEX_DB]

'''
读取paper数据库中的数据
'''
def get_db_data(university_name):
    # table = db[university_name]
    # data = pd.DataFrame(list(table.find()))
    data = db[university_name].find({}).batch_size(1)
    return data

'''
获取gender_sex函数
'''
def get_gender_sex_info(data_item,university_name):
    researcher_affiliation = data_item.get("AfN")
    researcher_name = data_item.get("AuN")
    print(researcher_affiliation)
    # 多线程去调 放在线程池
    get_aminer_data = do_get_aminer_data(researcher_name,researcher_affiliation)
    aminer_data_json,status = get_aminer_data.do_connect()
    print("aminer_data_json:",aminer_data_json)
    if aminer_data_json:
        gender_info = aminer_data_json.get("Final")
        print(gender_info)
        if gender_info.get("gender") == 'male':
            return None
        else:
            result = db[university_name].update({"AuN":researcher_name,"AfN":researcher_affiliation},{"$set":{"Sex":0}})
            if result:
                return None
            else:
                print("修改数据库中的数据出问题")
                return {"AuN":researcher_name,"AfN":researcher_affiliation}
    else:
        print("获取aminer数据失败")
        return {"AuN":researcher_name,"AfN":researcher_affiliation}

'''
对每个researcher数据进行操作，添加sex这一个字段
'''
def deal_researcher_sex_info(university_name):
    data = get_db_data(university_name)
    fail_check_sex_list = []
    i = 0
    for data_item in data:
        # print(data_item)
        print("--------------------------------------:",i)
        i += 1
        get_gender_sex = get_gender_sex_info(data_item,university_name)
        print("get_gender_sex:",get_gender_sex)
        if get_gender_sex:
            fail_check_sex_list.append(get_gender_sex)
            print(fail_check_sex_list)
    return fail_check_sex_list

'''
获取到所有的失败的数据
'''
def deal_fail_check_sex_list(fail_check_sex_list,university_name):
    list = []
    for data_item in fail_check_sex_list:
        get_gender_sex = get_gender_sex_info(data_item,university_name)
        if get_gender_sex:
            list.append(get_gender_sex)
    return list

'''
将所有失败的数据存入json文件中
'''
def save_fail_data_into_json(all_fail_list):
    all_fail_list_two = []
    for data_item in all_fail_list:
        for data in data_item:
            all_fail_list_two.append(data)
    f = codecs.open("fail_list.json",'a','utf-8')
    for data_item_two in all_fail_list_two:
        f.write(data_item_two)

    f.close()


'''
主函数
'''
def main():
    pool = Pool(processes = 4)
    all_fail_list = []
    all_fail_list.append(pool.map(deal_researcher_sex_info,UNIVERSITY_LIST))
    save_fail_data_into_json(all_fail_list)

    # university_name = UNIVERSITY_LIST
    # fail_check_sex_list = deal_researcher_sex_info(university_name[0])
    # while len(fail_check_sex_list) != 0:
    #     fail_check_sex_list = deal_fail_check_sex_list(fail_check_sex_list,university_name)

if __name__ == '__main__':
    main()
